{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# Using TensorFlow for Stylenet/NeuralStyle\n",
    "---------------------------------------\n",
    "\n",
    "We use two images, an original image and a style image and try to make the original image in the style of the style image.\n",
    "\n",
    "Reference paper:\n",
    "https://arxiv.org/abs/1508.06576\n",
    "\n",
    "Need to download the model 'imagenet-vgg-verydee-19.mat' from:\n",
    "http://www.vlfeat.org/matconvnet/models/beta16/imagenet-vgg-verydeep-19.mat\n",
    "\n",
    "We start by loading the necessary libraries and clearing any prior computational graph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import scipy.misc\n",
    "import scipy.io\n",
    "import numpy as np\n",
    "import tensorflow as tf\n",
    "from tensorflow.python.framework import ops\n",
    "ops.reset_default_graph()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Start a graph session"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sess = tf.Session()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Load image files and set algorithm parameters and weights."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Image Files\n",
    "original_image_file = 'temp/book_cover.jpg'\n",
    "style_image_file = 'temp/starry_night.jpg'\n",
    "\n",
    "# Saved VGG Network path\n",
    "vgg_path = '/home/nick/Documents/tensorflow/vgg_19_models/imagenet-vgg-verydeep-19.mat'\n",
    "\n",
    "# Default Arguments\n",
    "original_image_weight = 5.0\n",
    "style_image_weight = 500.0\n",
    "regularization_weight = 100\n",
    "learning_rate = 0.001\n",
    "generations = 5000\n",
    "output_generations = 250\n",
    "beta1 = 0.9   # For the Adam optimizer\n",
    "beta2 = 0.999 # For the Adam optimizer"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Read in the images."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Read in images\n",
    "original_image = scipy.misc.imread(original_image_file)\n",
    "style_image = scipy.misc.imread(style_image_file)\n",
    "\n",
    "# Get shape of target and make the style image the same\n",
    "target_shape = original_image.shape\n",
    "style_image = scipy.misc.imresize(style_image, target_shape[1] / style_image.shape[1])"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "VGG-19 Layer setup (From the paper https://arxiv.org/abs/1508.06576 )"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 22,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# VGG-19 Layer Setup\n",
    "# From paper\n",
    "vgg_layers = ['conv1_1', 'relu1_1',\n",
    "              'conv1_2', 'relu1_2', 'pool1',\n",
    "              'conv2_1', 'relu2_1',\n",
    "              'conv2_2', 'relu2_2', 'pool2',\n",
    "              'conv3_1', 'relu3_1',\n",
    "              'conv3_2', 'relu3_2',\n",
    "              'conv3_3', 'relu3_3',\n",
    "              'conv3_4', 'relu3_4', 'pool3',\n",
    "              'conv4_1', 'relu4_1',\n",
    "              'conv4_2', 'relu4_2',\n",
    "              'conv4_3', 'relu4_3',\n",
    "              'conv4_4', 'relu4_4', 'pool4',\n",
    "              'conv5_1', 'relu5_1',\n",
    "              'conv5_2', 'relu5_2',\n",
    "              'conv5_3', 'relu5_3',\n",
    "              'conv5_4', 'relu5_4']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Extract weights and matrix means"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 23,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def extract_net_info(path_to_params):\n",
    "    vgg_data = scipy.io.loadmat(path_to_params)\n",
    "    normalization_matrix = vgg_data['normalization'][0][0][0]\n",
    "    mat_mean = np.mean(normalization_matrix, axis=(0,1))\n",
    "    network_weights = vgg_data['layers'][0]\n",
    "    return(mat_mean, network_weights)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Create the VGG-19 Network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "def vgg_network(network_weights, init_image):\n",
    "    network = {}\n",
    "    image = init_image\n",
    "\n",
    "    for i, layer in enumerate(vgg_layers):\n",
    "        if layer[0] == 'c':\n",
    "            weights, bias = network_weights[i][0][0][0][0]\n",
    "            weights = np.transpose(weights, (1, 0, 2, 3))\n",
    "            bias = bias.reshape(-1)\n",
    "            conv_layer = tf.nn.conv2d(image, tf.constant(weights), (1, 1, 1, 1), 'SAME')\n",
    "            image = tf.nn.bias_add(conv_layer, bias)\n",
    "        elif layer[0] == 'r':\n",
    "            image = tf.nn.relu(image)\n",
    "        else:\n",
    "            image = tf.nn.max_pool(image, (1, 2, 2, 1), (1, 2, 2, 1), 'SAME')\n",
    "        network[layer] = image\n",
    "    return(network)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Here we define which layers apply to the original or style image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "original_layer = 'relu4_2'\n",
    "style_layers = ['relu1_1', 'relu2_1', 'relu3_1', 'relu4_1', 'relu5_1']"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get network parameters"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "normalization_mean, network_weights = extract_net_info(vgg_path)\n",
    "\n",
    "shape = (1,) + original_image.shape\n",
    "style_shape = (1,) + style_image.shape\n",
    "original_features = {}\n",
    "style_features = {}"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define placeholder and VGG network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 27,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "image = tf.placeholder('float', shape=shape)\n",
    "vgg_net = vgg_network(network_weights, image)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Normalize original image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 28,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "original_minus_mean = original_image - normalization_mean\n",
    "original_norm = np.array([original_minus_mean])\n",
    "original_features[original_layer] = sess.run(vgg_net[original_layer], feed_dict={image: original_norm})"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Get style image network"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 29,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "image = tf.placeholder('float', shape=style_shape)\n",
    "vgg_net = vgg_network(network_weights, image)\n",
    "style_minus_mean = style_image - normalization_mean\n",
    "style_norm = np.array([style_minus_mean])\n",
    "\n",
    "for layer in style_layers:\n",
    "    layer_output = sess.run(vgg_net[layer], feed_dict={image: style_norm})\n",
    "    layer_output = np.reshape(layer_output, (-1, layer_output.shape[3]))\n",
    "    style_gram_matrix = np.matmul(layer_output.T, layer_output) / layer_output.size\n",
    "    style_features[layer] = style_gram_matrix"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Make Combined Image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 30,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "initial = tf.random_normal(shape) * 0.256\n",
    "image = tf.Variable(initial)\n",
    "vgg_net = vgg_network(network_weights, image)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Define our loss as (Style Loss + Total Variation Loss)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 31,
   "metadata": {
    "collapsed": false
   },
   "outputs": [],
   "source": [
    "# Loss\n",
    "original_loss = original_image_weight * (2 * tf.nn.l2_loss(vgg_net[original_layer] - original_features[original_layer]) /\n",
    "                original_features[original_layer].size)\n",
    "\n",
    "# Loss from Style Image\n",
    "style_loss = 0\n",
    "style_losses = []\n",
    "for style_layer in style_layers:\n",
    "    layer = vgg_net[style_layer]\n",
    "    feats, height, width, channels = [x.value for x in layer.get_shape()]\n",
    "    size = height * width * channels\n",
    "    features = tf.reshape(layer, (-1, channels))\n",
    "    style_gram_matrix = tf.matmul(tf.transpose(features), features) / size\n",
    "    style_expected = style_features[style_layer]\n",
    "    #style_temp_loss = sess.run(2 * tf.nn.l2_loss(style_gram_matrix - style_expected) / style_expected.size)\n",
    "    #print('Layer: {}, Loss: {}'.format(style_layer, style_temp_loss))\n",
    "    style_losses.append(2 * tf.nn.l2_loss(style_gram_matrix - style_expected) / style_expected.size)\n",
    "style_loss += style_image_weight * tf.reduce_sum(style_losses)\n",
    "\n",
    "# To Smooth the resuts, we add in total variation loss       \n",
    "total_var_x = sess.run(tf.reduce_prod(image[:,1:,:,:].get_shape()))\n",
    "total_var_y = sess.run(tf.reduce_prod(image[:,:,1:,:].get_shape()))\n",
    "first_term = regularization_weight * 2\n",
    "second_term_numerator = tf.nn.l2_loss(image[:,1:,:,:] - image[:,:shape[1]-1,:,:])\n",
    "second_term = second_term_numerator / total_var_y\n",
    "third_term = (tf.nn.l2_loss(image[:,:,1:,:] - image[:,:,:shape[2]-1,:]) / total_var_x)\n",
    "total_variation_loss = first_term * (second_term + third_term)\n",
    "\n",
    "# Combined Loss\n",
    "loss = original_loss + style_loss + total_variation_loss"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Now we extract the style layer information"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 32,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "style_layer = 'relu2_1'\n",
    "layer = vgg_net[style_layer]\n",
    "feats, height, width, channels = [x.value for x in layer.get_shape()]\n",
    "size = height * width * channels\n",
    "features = tf.reshape(layer, (-1, channels))\n",
    "style_gram_matrix = tf.matmul(tf.transpose(features), features) / size\n",
    "style_expected = style_features[style_layer]\n",
    "style_losses.append(2 * tf.nn.l2_loss(style_gram_matrix - style_expected) / style_expected.size)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Declare optimization and initialize the variables in the graph."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 33,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# Declare Optimization Algorithm\n",
    "optimizer = tf.train.AdamOptimizer(learning_rate, beta1, beta2)\n",
    "train_step = optimizer.minimize(loss)\n",
    "\n",
    "# Initialize Variables and start Training\n",
    "sess.run(tf.global_variables_initializer())"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "For sanity and checking purposes, let's look at the matrices of intermediate images for the style layers."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 34,
   "metadata": {
    "collapsed": false,
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "-------Layer: relu1_1 -------\n",
      "[[[[  5.46172917e-01   9.92441326e-02   1.65350467e-01 ...,\n",
      "      1.01470089e+00   6.88668132e-01   6.07679486e-01]\n",
      "   [  1.04633188e+00   1.42677456e-01   2.02847883e-01 ...,\n",
      "      1.04049456e+00   4.46966410e-01   5.67616999e-01]\n",
      "   [  9.44447458e-01   9.57916826e-02   5.35283238e-02 ...,\n",
      "      1.01360738e+00   4.82194543e-01   4.57370222e-01]\n",
      "   ..., \n",
      "   [  7.98409760e-01   2.62508690e-01   1.63674399e-01 ...,\n",
      "      1.08287823e+00   6.82613015e-01   5.21779716e-01]\n",
      "   [  2.50822067e-01   2.98981339e-01   1.77775383e-01 ...,\n",
      "      9.34505463e-01   7.71890640e-01   4.95255679e-01]\n",
      "   [  7.59016812e-01   2.45658815e-01   2.49062672e-01 ...,\n",
      "      9.47369337e-01   4.31696802e-01   5.18461406e-01]]\n",
      "\n",
      "  [[  3.50127578e-01   6.55320138e-02   1.13581538e-01 ...,\n",
      "      8.30636561e-01   5.90469778e-01   4.08085167e-01]\n",
      "   [  9.41003978e-01   1.56615585e-01   1.50822163e-01 ...,\n",
      "      8.58183980e-01   2.27086306e-01   3.19344580e-01]\n",
      "   [  7.40134895e-01   1.21932805e-01   0.00000000e+00 ...,\n",
      "      8.26031208e-01   7.21089542e-02   0.00000000e+00]\n",
      "   ..., \n",
      "   [  7.85333812e-01   1.55980662e-01   1.17886856e-01 ...,\n",
      "      1.13494742e+00   7.07993031e-01   5.43884993e-01]\n",
      "   [  3.82750571e-01   2.00911433e-01   1.33004218e-01 ...,\n",
      "      8.91037047e-01   9.77851689e-01   6.41926885e-01]\n",
      "   [  1.08864868e+00   1.88541815e-01   3.56905192e-01 ...,\n",
      "      9.44619894e-01   8.96699905e-01   1.03038692e+00]]\n",
      "\n",
      "  [[  4.33232725e-01   8.70579258e-02   1.51543722e-01 ...,\n",
      "      9.45773184e-01   6.92379773e-01   6.00517273e-01]\n",
      "   [  9.22134459e-01   2.05127358e-01   1.83490515e-01 ...,\n",
      "      9.22428370e-01   3.70060325e-01   4.70131159e-01]\n",
      "   [  8.43977213e-01   2.13223189e-01   1.15390152e-01 ...,\n",
      "      9.68686581e-01   3.00063938e-01   2.99528390e-01]\n",
      "   ..., \n",
      "   [  4.56601799e-01   3.12734358e-02   0.00000000e+00 ...,\n",
      "      1.18501759e+00   3.01908374e-01   1.93765372e-01]\n",
      "   [  3.34473938e-01   1.12404957e-01   0.00000000e+00 ...,\n",
      "      9.44534302e-01   5.50019085e-01   2.69432098e-01]\n",
      "   [  8.31018209e-01   1.57822490e-01   2.40948036e-01 ...,\n",
      "      1.05588543e+00   8.67263675e-01   8.80930781e-01]]\n",
      "\n",
      "  ..., \n",
      "  [[  5.31517982e-01   0.00000000e+00   4.03768644e-02 ...,\n",
      "      8.78451824e-01   8.93990695e-01   7.68438816e-01]\n",
      "   [  1.19805670e+00   0.00000000e+00   1.31644428e-01 ...,\n",
      "      8.50356936e-01   6.92384958e-01   8.66060913e-01]\n",
      "   [  1.09240174e+00   6.70394227e-02   8.14481527e-02 ...,\n",
      "      9.53954816e-01   6.62642419e-02   1.46454036e-01]\n",
      "   ..., \n",
      "   [  8.64909053e-01   1.56760842e-01   1.55394629e-01 ...,\n",
      "      9.85976219e-01   4.75623906e-01   4.36004400e-01]\n",
      "   [  9.18938935e-01   6.20968230e-02   7.30036348e-02 ...,\n",
      "      1.11885953e+00   6.34442270e-02   1.61275312e-01]\n",
      "   [  6.47076070e-01   4.94428053e-02   1.01289526e-03 ...,\n",
      "      1.17038870e+00   2.11927891e-01   5.25588393e-02]]\n",
      "\n",
      "  [[  6.73230410e-01   0.00000000e+00   2.87192389e-02 ...,\n",
      "      9.60328817e-01   7.58015990e-01   6.46893203e-01]\n",
      "   [  6.88893080e-01   0.00000000e+00   3.64218652e-02 ...,\n",
      "      8.96826148e-01   4.80435818e-01   4.48961675e-01]\n",
      "   [  6.72059238e-01   7.46200308e-02   9.21256691e-02 ...,\n",
      "      9.38762903e-01   2.85537809e-01   2.79351294e-01]\n",
      "   ..., \n",
      "   [  1.09793949e+00   2.33404368e-01   2.27619678e-01 ...,\n",
      "      1.16681099e+00   8.93943310e-01   9.28698003e-01]\n",
      "   [  1.23093176e+00   9.29298699e-02   1.32145494e-01 ...,\n",
      "      1.18841565e+00   5.66266179e-01   6.02929592e-01]\n",
      "   [  5.80879629e-01   2.68148333e-02   5.22332080e-03 ...,\n",
      "      1.16700470e+00   4.37579751e-01   3.01933765e-01]]\n",
      "\n",
      "  [[  6.19279444e-01   5.71221113e-03   7.33239204e-03 ...,\n",
      "      1.02036691e+00   6.15148008e-01   4.60092902e-01]\n",
      "   [  6.64646208e-01   5.14058992e-02   1.40249133e-01 ...,\n",
      "      1.02806544e+00   8.03461909e-01   7.28130162e-01]\n",
      "   [  1.07668614e+00   1.39771670e-01   2.16293976e-01 ...,\n",
      "      9.61770236e-01   6.48612142e-01   6.86056435e-01]\n",
      "   ..., \n",
      "   [  9.18366194e-01   1.51607066e-01   1.01785958e-01 ...,\n",
      "      1.11267006e+00   4.23615158e-01   3.66022974e-01]\n",
      "   [  7.29683459e-01   4.01226319e-02   4.56322879e-02 ...,\n",
      "      1.11998713e+00   4.78762478e-01   2.91840732e-01]\n",
      "   [  5.69028795e-01   3.23730558e-02   6.10724315e-02 ...,\n",
      "      1.17132306e+00   6.05213523e-01   4.71167862e-01]]]]\n",
      "9555712\n",
      "\n",
      "-------Layer: relu2_1 -------\n",
      "[[[[  0.00000000e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      4.17218876e+00   0.00000000e+00   9.70013618e+00]\n",
      "   [  0.00000000e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      3.36537862e+00   0.00000000e+00   1.35666962e+01]\n",
      "   [  0.00000000e+00   9.06453848e-01   0.00000000e+00 ...,\n",
      "      6.48856020e+00   0.00000000e+00   1.45966320e+01]\n",
      "   ..., \n",
      "   [  0.00000000e+00   7.25086689e-01   0.00000000e+00 ...,\n",
      "      5.46153450e+00   0.00000000e+00   1.47363615e+01]\n",
      "   [  0.00000000e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      4.45307255e+00   0.00000000e+00   1.27726164e+01]\n",
      "   [  0.00000000e+00   1.87854695e+00   0.00000000e+00 ...,\n",
      "      7.86684704e+00   0.00000000e+00   9.70730686e+00]]\n",
      "\n",
      "  [[  0.00000000e+00   0.00000000e+00   7.62732267e-01 ...,\n",
      "      4.45038557e+00   0.00000000e+00   1.88567603e+00]\n",
      "   [  0.00000000e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      4.10391903e+00   1.46153796e+00   2.57139802e+00]\n",
      "   [  0.00000000e+00   0.00000000e+00   1.80198163e-01 ...,\n",
      "      5.97354841e+00   0.00000000e+00   1.95740902e+00]\n",
      "   ..., \n",
      "   [  0.00000000e+00   8.45689416e-01   0.00000000e+00 ...,\n",
      "      5.62061596e+00   0.00000000e+00   2.12175679e+00]\n",
      "   [  0.00000000e+00   2.04315573e-01   0.00000000e+00 ...,\n",
      "      3.42241049e+00   0.00000000e+00   1.72085202e+00]\n",
      "   [  0.00000000e+00   6.61905825e-01   1.60468709e+00 ...,\n",
      "      6.76005840e+00   0.00000000e+00   2.58036113e+00]]\n",
      "\n",
      "  [[  0.00000000e+00   0.00000000e+00   1.01503921e+00 ...,\n",
      "      3.86895227e+00   0.00000000e+00   3.57661653e+00]\n",
      "   [  0.00000000e+00   0.00000000e+00   4.92518872e-01 ...,\n",
      "      4.65180588e+00   0.00000000e+00   3.53237557e+00]\n",
      "   [  0.00000000e+00   0.00000000e+00   7.66881704e-01 ...,\n",
      "      6.56707954e+00   0.00000000e+00   1.54645836e+00]\n",
      "   ..., \n",
      "   [  0.00000000e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      8.82021809e+00   0.00000000e+00   3.50165486e+00]\n",
      "   [  0.00000000e+00   3.07039332e+00   0.00000000e+00 ...,\n",
      "      1.94095957e+00   0.00000000e+00   4.75953102e+00]\n",
      "   [  0.00000000e+00   1.02102757e+00   1.84634417e-01 ...,\n",
      "      3.17570853e+00   0.00000000e+00   3.64062238e+00]]\n",
      "\n",
      "  ..., \n",
      "  [[  0.00000000e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      3.21217036e+00   0.00000000e+00   3.56954217e+00]\n",
      "   [  0.00000000e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      5.09261656e+00   6.50575697e-01   2.71042085e+00]\n",
      "   [  0.00000000e+00   2.36166984e-01   0.00000000e+00 ...,\n",
      "      9.62703133e+00   0.00000000e+00   1.27424824e+00]\n",
      "   ..., \n",
      "   [  0.00000000e+00   3.98477030e+00   0.00000000e+00 ...,\n",
      "      6.54567337e+00   2.20176673e+00   0.00000000e+00]\n",
      "   [  0.00000000e+00   3.56805533e-01   0.00000000e+00 ...,\n",
      "      2.97434616e+00   1.53505409e+00   0.00000000e+00]\n",
      "   [  0.00000000e+00   2.07912564e+00   3.51652592e-01 ...,\n",
      "      5.31449938e+00   1.17559302e+00   6.78300083e-01]]\n",
      "\n",
      "  [[  0.00000000e+00   0.00000000e+00   8.68020773e-01 ...,\n",
      "      8.61162663e+00   0.00000000e+00   0.00000000e+00]\n",
      "   [  0.00000000e+00   4.44363132e-02   0.00000000e+00 ...,\n",
      "      4.32354546e+00   2.42411947e+00   0.00000000e+00]\n",
      "   [  0.00000000e+00   0.00000000e+00   1.87926555e+00 ...,\n",
      "      9.69424152e+00   0.00000000e+00   0.00000000e+00]\n",
      "   ..., \n",
      "   [  0.00000000e+00   4.06546402e+00   3.39299649e-01 ...,\n",
      "      6.75921726e+00   1.32584274e+00   9.43677127e-03]\n",
      "   [  0.00000000e+00   2.19557071e+00   0.00000000e+00 ...,\n",
      "      6.37346983e+00   2.66051340e+00   2.68892467e-01]\n",
      "   [  0.00000000e+00   2.46153617e+00   4.17754620e-01 ...,\n",
      "      4.90830708e+00   1.38570654e+00   8.22443664e-01]]\n",
      "\n",
      "  [[  0.00000000e+00   0.00000000e+00   4.14864731e+00 ...,\n",
      "      4.97171211e+00   9.50375617e-01   0.00000000e+00]\n",
      "   [  0.00000000e+00   1.38943148e+00   3.29794145e+00 ...,\n",
      "      9.83230686e+00   6.20348835e+00   0.00000000e+00]\n",
      "   [  0.00000000e+00   0.00000000e+00   2.85489559e+00 ...,\n",
      "      5.73570442e+00   4.86837292e+00   0.00000000e+00]\n",
      "   ..., \n",
      "   [  0.00000000e+00   1.80851126e+00   4.49450636e+00 ...,\n",
      "      6.30092812e+00   4.62885618e+00   0.00000000e+00]\n",
      "   [  0.00000000e+00   1.29729223e+00   4.71223927e+00 ...,\n",
      "      4.89820623e+00   2.10295463e+00   0.00000000e+00]\n",
      "   [  0.00000000e+00   1.79755044e+00   3.02188969e+00 ...,\n",
      "      4.78452396e+00   1.55736673e+00   0.00000000e+00]]]]\n",
      "4777856\n",
      "\n",
      "-------Layer: relu3_1 -------\n",
      "[[[[  1.28948152e+00   1.09039581e+00   0.00000000e+00 ...,\n",
      "      2.19649136e-01   5.20211172e+00   1.13432322e+01]\n",
      "   [  6.05245903e-02   7.80232966e-01   2.29341269e+00 ...,\n",
      "      0.00000000e+00   4.88790178e+00   1.14025221e+01]\n",
      "   [  1.26504219e+00   4.87358332e+00   1.88070524e+00 ...,\n",
      "      1.21586323e+00   4.90619421e+00   9.36695480e+00]\n",
      "   ..., \n",
      "   [  0.00000000e+00   1.22829400e-01   2.41078687e+00 ...,\n",
      "      1.33314681e+00   1.41576090e+01   9.20385933e+00]\n",
      "   [  1.84576929e+00   1.73521149e+00   9.43663836e-01 ...,\n",
      "      0.00000000e+00   1.54128809e+01   4.44435072e+00]\n",
      "   [  5.05310535e-01   3.49539876e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   9.92387772e+00   2.31832695e+00]]\n",
      "\n",
      "  [[  1.33337963e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   5.18965721e+00   1.22029991e+01]\n",
      "   [  7.82827735e-01   0.00000000e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   3.04198050e+00   9.93384647e+00]\n",
      "   [  1.31841958e+00   4.93920612e+00   0.00000000e+00 ...,\n",
      "      1.83317792e+00   0.00000000e+00   4.47278214e+00]\n",
      "   ..., \n",
      "   [  4.57008076e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   7.04834557e+00   3.98935485e+00]\n",
      "   [  6.55452299e+00   0.00000000e+00   4.17389125e-01 ...,\n",
      "      0.00000000e+00   1.05839224e+01   0.00000000e+00]\n",
      "   [  1.41748178e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   8.54816055e+00   4.67239320e-03]]\n",
      "\n",
      "  [[  0.00000000e+00   1.61564663e-01   0.00000000e+00 ...,\n",
      "      0.00000000e+00   3.07246590e+00   1.61753712e+01]\n",
      "   [  9.09780502e-01   0.00000000e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   3.54566431e+00   1.04304495e+01]\n",
      "   [  2.60656691e+00   3.37819767e+00   0.00000000e+00 ...,\n",
      "      2.37193441e+00   0.00000000e+00   6.64558125e+00]\n",
      "   ..., \n",
      "   [  7.38889647e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   4.71350861e+00   5.79951286e+00]\n",
      "   [  6.76442957e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   2.49481964e+00   0.00000000e+00]\n",
      "   [  2.35785079e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   3.32504392e+00   0.00000000e+00]]\n",
      "\n",
      "  ..., \n",
      "  [[  7.29322791e-01   4.31529135e-01   0.00000000e+00 ...,\n",
      "      2.74690127e+00   9.71999550e+00   1.89540119e+01]\n",
      "   [  5.98677301e+00   0.00000000e+00   1.50127351e+00 ...,\n",
      "      4.78056526e+00   3.57778519e-02   8.89444065e+00]\n",
      "   [  0.00000000e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      1.24823403e+00   0.00000000e+00   8.16132069e+00]\n",
      "   ..., \n",
      "   [  2.62365365e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   1.65535755e+01   9.10090637e+00]\n",
      "   [  4.37235403e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   1.07394371e+01   4.82493114e+00]\n",
      "   [  2.45628333e+00   8.54965746e-01   0.00000000e+00 ...,\n",
      "      1.77847624e-01   8.02072239e+00   0.00000000e+00]]\n",
      "\n",
      "  [[  1.21734369e+00   2.47041300e-01   0.00000000e+00 ...,\n",
      "      0.00000000e+00   5.11761129e-01   1.96734924e+01]\n",
      "   [  1.73002374e+00   0.00000000e+00   3.59340400e-01 ...,\n",
      "      0.00000000e+00   0.00000000e+00   8.95430756e+00]\n",
      "   [  1.12477615e-02   8.73840630e-01   0.00000000e+00 ...,\n",
      "      0.00000000e+00   0.00000000e+00   7.18093634e+00]\n",
      "   ..., \n",
      "   [  3.96339035e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   1.02852182e+01   9.44911861e+00]\n",
      "   [  5.64176989e+00   0.00000000e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   7.89662504e+00   3.73874593e+00]\n",
      "   [  1.86112893e+00   8.96617472e-01   0.00000000e+00 ...,\n",
      "      0.00000000e+00   6.92846870e+00   0.00000000e+00]]\n",
      "\n",
      "  [[  0.00000000e+00   1.92645681e+00   0.00000000e+00 ...,\n",
      "      0.00000000e+00   0.00000000e+00   1.45687075e+01]\n",
      "   [  0.00000000e+00   1.79749382e+00   1.71754038e+00 ...,\n",
      "      2.01468676e-01   0.00000000e+00   1.03209276e+01]\n",
      "   [  0.00000000e+00   5.73547316e+00   0.00000000e+00 ...,\n",
      "      1.53854001e+00   0.00000000e+00   8.76346493e+00]\n",
      "   ..., \n",
      "   [  0.00000000e+00   0.00000000e+00   1.53463590e+00 ...,\n",
      "      0.00000000e+00   0.00000000e+00   1.11891747e+01]\n",
      "   [  0.00000000e+00   1.37989533e+00   3.74676228e-01 ...,\n",
      "      5.86541533e-01   0.00000000e+00   6.89601517e+00]\n",
      "   [  0.00000000e+00   1.96457899e+00   0.00000000e+00 ...,\n",
      "      2.85209775e-01   4.47431982e-01   1.64669716e+00]]]]\n",
      "2414080\n",
      "\n",
      "-------Layer: relu4_1 -------\n",
      "[[[[  0.           8.06938171   0.         ...,   0.          35.53449631\n",
      "      0.        ]\n",
      "   [ 10.79806328  21.55803108   0.         ...,   0.          18.4797039\n",
      "      0.        ]\n",
      "   [ 15.51786327  22.00497818   0.         ...,   0.          22.2850914\n",
      "     11.4221344 ]\n",
      "   ..., \n",
      "   [  9.37768078  20.40073204   0.         ...,   0.          17.54686165\n",
      "      9.24625778]\n",
      "   [  0.          32.72386551   0.         ...,   0.           0.\n",
      "     11.4426899 ]\n",
      "   [  0.          32.14693451  11.02395916 ...,   0.           0.\n",
      "     24.20874214]]\n",
      "\n",
      "  [[ 18.09027863  14.80698013   1.41294694 ...,   0.          46.34196854\n",
      "      0.        ]\n",
      "   [  8.3393259   30.82091141  11.86311722 ...,   0.           5.61054707\n",
      "      0.        ]\n",
      "   [  6.0640831   32.61615372   4.90172148 ...,   0.          13.45902443\n",
      "      0.        ]\n",
      "   ..., \n",
      "   [  0.          33.23873138   0.         ...,   0.          26.33929062\n",
      "      0.        ]\n",
      "   [  0.          51.44049454   9.45747471 ...,   0.           0.           0.        ]\n",
      "   [  0.          49.19579697  31.04375458 ...,   0.           0.\n",
      "     20.10377121]]\n",
      "\n",
      "  [[ 12.66357708  21.13592148   2.92949677 ...,   0.          55.29376602\n",
      "      0.        ]\n",
      "   [  0.          33.722332    11.56882286 ...,   0.          12.36106873\n",
      "      0.        ]\n",
      "   [  0.          29.61502266   0.42656589 ...,   0.          10.62097549\n",
      "      0.        ]\n",
      "   ..., \n",
      "   [  0.          26.6602478    0.         ...,   0.          33.44818497\n",
      "      0.        ]\n",
      "   [  0.          43.49346542   3.72418356 ...,   0.           0.\n",
      "      6.22570515]\n",
      "   [  1.93495631  43.91726685  23.80758476 ...,   0.           0.\n",
      "     22.9033947 ]]\n",
      "\n",
      "  ..., \n",
      "  [[  8.90740395  11.15606308   0.         ...,   0.          51.58481216\n",
      "      0.        ]\n",
      "   [  0.          27.82268333   0.         ...,   0.          17.93111801\n",
      "      0.        ]\n",
      "   [  5.89332628  27.88334656   0.         ...,   0.          10.31050491\n",
      "      0.        ]\n",
      "   ..., \n",
      "   [  0.86502963  36.57086182   0.         ...,   0.          31.35421944\n",
      "      0.        ]\n",
      "   [  0.          41.98275375   5.13476133 ...,   0.           0.\n",
      "      0.53653204]\n",
      "   [  0.          45.85266876  20.92389297 ...,   0.           0.\n",
      "     21.07262993]]\n",
      "\n",
      "  [[  7.39349365   6.77328205   0.         ...,   0.          51.82925797\n",
      "      0.        ]\n",
      "   [  0.          27.02615738   0.         ...,   0.          29.89388657\n",
      "      0.        ]\n",
      "   [  0.          26.54285812   0.         ...,   0.          13.83098602\n",
      "      0.        ]\n",
      "   ..., \n",
      "   [  0.          37.77023315   0.         ...,   0.          29.97325134\n",
      "      0.        ]\n",
      "   [  0.          39.09495926   3.84527183 ...,   0.           0.           0.        ]\n",
      "   [  0.          45.01422501  20.30278778 ...,   0.           0.\n",
      "     15.57155323]]\n",
      "\n",
      "  [[ 34.4567337    1.02669334   4.33333063 ...,   0.          33.06891251\n",
      "      0.        ]\n",
      "   [  5.72960329  16.96014595  13.27928734 ...,   0.          23.35857582\n",
      "      0.        ]\n",
      "   [  0.24360615  18.50901413   5.88972902 ...,   0.          10.86125278\n",
      "      0.        ]\n",
      "   ..., \n",
      "   [  0.          32.99405289  14.02725124 ...,   0.          23.79265404\n",
      "      0.        ]\n",
      "   [  0.          29.20583916  21.73028946 ...,   0.           0.\n",
      "      3.53504109]\n",
      "   [  0.          31.97105408  25.80918121 ...,   0.           0.\n",
      "      9.19038963]]]]\n",
      "1217536\n",
      "\n",
      "-------Layer: relu5_1 -------\n",
      "[[[[ 1.70734537  0.          4.05844355 ...,  0.84914148  0.          0.        ]\n",
      "   [ 2.01084232  0.          3.16427231 ...,  0.          0.          0.        ]\n",
      "   [ 1.01718831  0.          1.440714   ...,  0.          0.          0.        ]\n",
      "   ..., \n",
      "   [ 0.55688125  0.          1.65481532 ...,  0.          0.          0.        ]\n",
      "   [ 0.          0.          4.7049346  ...,  0.          0.          0.        ]\n",
      "   [ 3.82551169  0.          3.46821284 ...,  0.          0.          0.        ]]\n",
      "\n",
      "  [[ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   ..., \n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   [ 1.4709177   0.          0.         ...,  0.          0.          0.        ]]\n",
      "\n",
      "  [[ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   ..., \n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]]\n",
      "\n",
      "  ..., \n",
      "  [[ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.96687579  0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.16698605  0.          0.        ]\n",
      "   ..., \n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]]\n",
      "\n",
      "  [[ 0.          0.          0.         ...,  2.07278061  0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  2.47546673  0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  1.14959514  0.          0.        ]\n",
      "   ..., \n",
      "   [ 0.          0.          0.         ...,  0.73376215  0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]]\n",
      "\n",
      "  [[ 0.          0.          0.         ...,  0.33846614  0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.81134415  0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.82347047  0.          0.        ]\n",
      "   ..., \n",
      "   [ 0.          0.          0.         ...,  1.10223734  0.          0.        ]\n",
      "   [ 0.          0.          0.         ...,  0.          0.          0.        ]\n",
      "   [ 0.17219058  0.          0.         ...,  0.          0.          0.        ]]]]\n",
      "311808\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for style_layer in style_layers:\n",
    "    print('-------Layer: {} -------'.format(style_layer))\n",
    "    layer = vgg_net[style_layer]\n",
    "    print(sess.run(layer))\n",
    "    feats, height, width, channels = [x.value for x in layer.get_shape()]\n",
    "    size = height * width * channels\n",
    "    print(size)\n",
    "    print('')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 35,
   "metadata": {
    "collapsed": false
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Generation 100 out of 1000, loss: 197868048.0\n",
      "Generation 200 out of 1000, loss: 105772656.0\n",
      "Generation 300 out of 1000, loss: 73410864.0\n",
      "Generation 400 out of 1000, loss: 57265060.0\n",
      "Generation 500 out of 1000, loss: 47589056.0\n",
      "Generation 600 out of 1000, loss: 41183492.0\n",
      "Generation 700 out of 1000, loss: 36635288.0\n",
      "Generation 800 out of 1000, loss: 33242754.0\n",
      "Generation 900 out of 1000, loss: 30612820.0\n",
      "Generation 1000 out of 1000, loss: 28516306.0\n"
     ]
    }
   ],
   "source": [
    "# Declare Optimization Algorithm\n",
    "optimizer = tf.train.GradientDescentOptimizer(learning_rate)\n",
    "train_step = optimizer.minimize(loss)\n",
    "\n",
    "# Initialize Variables and start Training\n",
    "sess.run(tf.global_variables_initializer())\n",
    "for i in range(generations):\n",
    "    \n",
    "    sess.run(train_step)\n",
    "\n",
    "    # Print update and save temporary output\n",
    "    if (i+1) % output_generations == 0:\n",
    "        print('Generation {} out of {}, loss: {}'.format(i + 1, generations,sess.run(loss)))\n",
    "        image_eval = sess.run(image)\n",
    "        best_image_add_mean = image_eval.reshape(shape[1:]) + normalization_mean\n",
    "        output_file = 'temp_output_{}.jpg'.format(i)\n",
    "        scipy.misc.imsave(output_file, best_image_add_mean)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Plot images:"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "Save final image"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "image_eval = sess.run(image)\n",
    "best_image_add_mean = image_eval.reshape(shape[1:]) + normalization_mean\n",
    "output_file = 'final_output.jpg'\n",
    "scipy.misc.imsave(output_file, best_image_add_mean)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.5.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
